import datetime
import os
import time

import logzero
import requests
from logzero import logger
from lxml import html
from tqdm import tqdm

import config
from utils import get_mongo_collection, record_run_status

COLLECTION_NAME = 'diyidan'
COLLECTION_FEED = 'feed'
db = get_mongo_collection(COLLECTION_NAME)
current_module = os.path.splitext(os.path.basename(__file__))[0]
logzero.logfile(f"/tmp/{current_module}.log", maxBytes=30_000_000, backupCount=3, encoding='utf-8')

insert_count = 0


# url = 'https://www.diyidan.com/user/posts/6294196636903201948/1'
def get_diyidan(url):
    next = True
    if requests.get(url, headers=config.HEADERS).url=='https://www.diyidan.com/':
        return False
    page = requests.get(url, headers=config.HEADERS).text
    pageinfo = html.fromstring(page)
    no_content = len(pageinfo.cssselect('.no_content'))
    if no_content:
        next = False
    else:
        div = pageinfo.cssselect('#perhezi > div > div > li ')
        divlen = len(div)
        for i in range(divlen):
            time.sleep(1.2)
            img = []
            bigImg = []
            infoUrl = 'https://www.diyidan.com' + div[i].get('onclick').split("'")[1]
            try:
                imgTmp = 'https:' + div[i].cssselect('.yuan_middle >.sheng_img > img')[0].get('src')
                bigImgTmp = imgTmp.replace('index', 'large')
            except IndexError:
                imgTmp = ''
                bigImgTmp = ''
            img.append(imgTmp)
            bigImg.append(bigImgTmp)
            title = div[i].cssselect('.hh')[0].text
            desp = div[i].cssselect('.ie2')[0].text
            username = div[i].cssselect('.yuan_top > .yuan_img > p ')[0].text
            timeStr = div[i].cssselect('.yuan_top > .yuan_img > span ')[0].text
            infoPage = html.fromstring(requests.get(infoUrl, headers=config.HEADERS).text)
            infoImg = infoPage.cssselect('.user_post_content >p >a > .if_error_show')
            video = len(infoPage.cssselect('video'))
            if not video:
                for j in infoImg:
                    infoImg = j.get('src')
                    infoImg = 'https:' + infoImg
                    if infoImg not in bigImg:
                        bigImg.append(infoImg)
                        img.append(infoImg.replace('large', 'index'))
            data = {}
            data['desp'] = desp
            data['is_video'] = video
            data['pic'] = img
            data['big_pic'] = bigImg
            data['title'] = title
            data['name'] = username
            data['time'] = timeStr
            data['href'] = infoUrl
            data['second'] = int(time.time())
            if store_data(data):
                return False
    return next


# //image.diyidan.net/post/2017/12/10/ggULG1bkmuqKCJxS.jpg!weblarge
# //image.diyidan.net/post/2017/12/10/ggULG1bkmuqKCJxS.jpg!webindex
def store_data(data):
    global insert_count
    stop = False
    aid = data['href']
    old_record = db.find_one({'href': aid})
    if old_record:
        logger.info(f'{aid} 已经存在，不再爬取')
        return True
    else:
        logger.info(f'{aid} 不存在，现在插入')
        data['update_time'] = datetime.datetime.now()
        logger.info(data)
        db.insert_one(data)
        insert_count += 1
        return False


def get_feeds():
    db_feed = get_mongo_collection(COLLECTION_FEED)
    cursor = db_feed.find({'purpose': '第一弹'})
    feeds = []
    for cu in cursor:
        feeds.append(cu['uid'])
    return feeds


@record_run_status(current_module)
def main():
    user = get_feeds()
    if not user:
        user = [6294196636855197848, 6294360860118920124, 6293615542267399811, 6293877835963977051, 6293877835972036139,
                6293877835977916171, 6293615542258106464, 6293615542285867173]
    for i in tqdm(user):
        page = 1
        next = True
        while next:
            next = get_diyidan('https://www.diyidan.com/user/posts/' + str(i) + '/' + str(page))
            page += 1
    return insert_count


if __name__ == '__main__':
    main()
